#!/usr/bin/python

import os
import numpy
from collections import Counter
from pylab import *
import tables,scipy
from scipy import fftpack
from scipy.cluster import vq
from scipy.ndimage import morphology,filters,interpolation
from ocrolib import docproc,improc
import pyflann
import random
import tables
from multiprocessing import Pool
import fcntl
from pdb import pm
from llpy.sutils import *
import unicodedata

import argparse
parser = argparse.ArgumentParser( description = "Show character statistics.")
parser.add_argument('input',default=None,help="data file")
parser.add_argument('-r','--norejects',action='store_true')
parser.add_argument('-m','--minperclass',type=int,default=50)
args = parser.parse_args()

with tables.openFile(args.input) as db:
    nsamples = len(db.root.classes)
    classes = array(db.root.classes[:nsamples],'i')
    distinct = Counter(classes)
    good = []
    kept = 0
    for cls,n in distinct.most_common(999999):
        c = udecode(cls)
        if c=="": continue
        if re.search('[\\000-\\037]',c): continue
        if "_" in c: continue
        if "~" in c and len(c)>1: continue
        if "~"==c and args.norejects: continue
        if n<args.minperclass: continue
        good.append(cls)
        print cls,n
        kept += n

print "classes",len(distinct.most_common()),len(good),"samples",nsamples,kept

if 0:
    for c in range(ord("a"),ord("z")+1): assert c in good
    for c in range(ord("A"),ord("Z")+1): assert c in good
    for c in range(ord("0"),ord("9")+1): assert c in good
    assert ord("_") not in good

updated = 0
with tables.openFile(args.input,"r+") as db:
    classes = db.root.classes
    _classes = db.createCArray(db.root,"classes0",tables.Int64Atom(),shape=(nsamples,),filters=tables.Filters(9))
    for i in range(nsamples):
        if classes[i] not in good: 
            _classes[i] = ord("_")
        else:
            _classes[i] = classes[i]
            updated += 1
    print "updated",updated
    # assert updated==kept
    db.flush()
    db.renameNode("/classes0","classes",overwrite=1)
    db.flush()
